package st.spider;

import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.MongoClient;

public class CatchOmdyPages implements PageProcessor {

	private Site site = Site.me().setDomain("www.xuandy.com").setSleepTime(15);

	@SuppressWarnings("deprecation")
	public void process(Page page) {

		List<Selectable> nodes = page.getHtml().xpath("//a[@rel='bookmark']")
				.nodes();

		String url = page.getUrl().toString();
		String[] parts = url.split("/");
		int index = Integer.parseInt(parts[parts.length - 1]);
		String tag = parts[parts.length - 3];

		List<BasicDBObject> os = new ArrayList<BasicDBObject>();
		for (Selectable node : nodes) {
			System.out.println(node.xpath("//a/@href").get());
			System.out.println(node.xpath("//a/text()").get());

			BasicDBObject temp = new BasicDBObject();
			temp.put("href", node.xpath("//a/@href").get());
			temp.put("title", node.xpath("//a/text()").get());
			os.add(temp);
		}

//		try {
//			MongoClient mongoClient = new MongoClient("115.159.74.171", 27017);
//			DB db = mongoClient.getDB("cult_site");
//			db.authenticate("root", new char[] { 'r', 'o', 'o', 't' });
//			DBCollection dbc = db.getCollection("page");
//
//			BasicDBObject o = new BasicDBObject();
//			o.put("tag", tag);
//			o.put("index", index);
//			o.put("items", os);
//			dbc.insert(o);
//		} catch (UnknownHostException e) {
//			e.printStackTrace();
//		}
	}

	public Site getSite() {
		return site;

	}

	public static void main(String[] args) {
		// prepare omdy pageitems
		for (int i = 1; i <= 354; i++) {
			Spider.create(new CatchOmdyPages())
					.addUrl("http://www.xuandy.com/category/movie/omdy/page/"
							+ i).addPipeline(new ConsolePipeline()).run();
		}
	}
}